﻿Imports System.Net
Imports System.Text.RegularExpressions
Imports System.IO
Public Class clsLinksFromURL
    Private Const _urlExp As String = "(http|https)://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?"
    Private Const _linkExpr As String = "href\s*=\s*(?:""(?<1>[^""]*)""|(?<1>\S+))"
    Private _target As String
    Private _wc As WebClient
    Private _content As String
    Private _targetUri As Uri
    Private _hrefLinks As New List(Of String)
    Private _httpLinks As New List(Of String)
    Private _links As New Dictionary(Of String, clsLink)
    Private _sizeRead As Long
    Public Sub New(Adresse As String)
        _target = Adresse
        Try
            _targetUri = New Uri(_target)
            If _validAdresse(_targetUri.LocalPath) Then
                _wc = New WebClient
                _content = _wc.DownloadString(_target)
                _sizeRead += _content.Length
                _extractHrefLinks()
                _extractHttpLinks()
                _collectLinks()
            End If
        Catch ex As Exception

        End Try
    End Sub
    Public ReadOnly Property Target As Uri
        Get
            Target = _targetUri
        End Get
    End Property
    Public ReadOnly Property Links As Dictionary(Of String, clsLink)
        Get
            Links = _links
        End Get
    End Property
    Public ReadOnly Property SizeRead As Long
        Get
            SizeRead = _sizeRead
        End Get
    End Property
    Private Function _validAdresse(url As String) As Boolean
        Dim ext As String

        _validAdresse = True
        ext = Path.GetExtension(url).ToLower
        If ext.Length <> 0 Then
            Select Case ext
                Case ".gif"
                    _validAdresse = False
                Case ".png"
                    _validAdresse = False
                Case ".ico"
                    _validAdresse = False
                Case ".dtd"
                    _validAdresse = False
                Case ".css"
                    _validAdresse = False
                Case ".xml"
                    _validAdresse = False
                Case ".cab"
                    _validAdresse = False
                Case ".jpg"
                    _validAdresse = False
                Case ".zip"
                    _validAdresse = False
                Case ".tgz"
                    _validAdresse = False
                Case ".pdf"
                    _validAdresse = False
                Case ".ps"
                    _validAdresse = False
                Case ".mp4"
                    _validAdresse = False
                Case ".mp3"
                    _validAdresse = False
                Case ".wma"
                    _validAdresse = False
                Case ".mov"
                    _validAdresse = False
                Case Else
                    _validAdresse = True
            End Select
        End If

    End Function
    Private Sub _collectLinks()
        Dim ls As String
        Dim u As Uri
        Dim link As clsLink

        For Each ls In _hrefLinks

            u = New Uri(_targetUri, ls)
            If u.AbsoluteUri.ToLower.Contains("http") Then
                If Not _links.ContainsKey(u.AbsoluteUri) Then
                    link = New clsLink(u.AbsoluteUri)
                    _links.Add(link.Url, link)
                End If
            End If
        Next

        For Each ls In _httpLinks
            u = New Uri(_targetUri, ls)
            If Not _links.ContainsKey(u.AbsoluteUri) Then
                link = New clsLink(u.AbsoluteUri)
                _links.Add(link.Url, link)
            End If
        Next

    End Sub
    Private Sub _extractHrefLinks()
        Dim reg As Regex
        Dim result As MatchCollection

        Dim s() As String
        reg = New Regex(_linkExpr, RegexOptions.IgnoreCase Or RegexOptions.ExplicitCapture)
        result = reg.Matches(_content)
        For Each m In result
            s = m.Value.Split("""")
            If s.Length = 3 Then
                _hrefLinks.Add(s(1))
            End If
        Next
    End Sub
    Private Sub _extractHttpLinks()
        Dim reg As Regex
        Dim result As MatchCollection

        reg = New Regex(_urlExp, RegexOptions.IgnoreCase Or RegexOptions.ExplicitCapture)
        result = reg.Matches(_content)
        For Each m In result
            _httpLinks.Add(m.Value)
        Next
    End Sub
End Class
